package com.pinfly.common.http;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.InetAddress;
import java.net.Socket;
import java.net.URL;
import java.net.UnknownHostException;
import java.security.ProviderException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;

import org.apache.log4j.Logger;

import com.pinfly.common.config.ConfigException;
import com.pinfly.common.config.PfyProperties;
import com.pinfly.common.config.PfyPropertiesFactory;
import com.pinfly.common.util.StringUtil;

/**
 * Class to send an HTTP request to a site and retrieve the response.
 * 
 * This class enables the user to specify an HTTP server, a resource on that
 * server and optionally the body of a request. The class will then send the
 * request to the server and return the result. If a response is not received in
 * a timely fashion, then another attempt is made. Both the time between
 * attempts and the number of attempts can be specified. A time of 0 indicates
 * that the request should never timeout.
 * <p>
 * Socket connections to the same server may be reused across multiple instances
 * of HttpClient.
 */
public class HttpClient
{
    private static final Logger logger = Logger.getLogger (HttpClient.class);

    // Default values for the configurable items.
    // These are used if the configurable items aren't specified or are invalid
    private final static int DEFAULT_REQUEST_TIMEOUT = 35000; // Wait 35 seconds
                                                              // for a response
    private final static int DEFAULT_REQUEST_RETRY_COUNT = 1; // Retry request
                                                              // one time
    private final static int DEFAULT_IDLE_TIMEOUT = 3500; // Wait 3.5 seconds
                                                          // for another
                                                          // request before
                                                          // closing socket.
                                                          // We want this
                                                          // timeout to be
                                                          // smaller
                                                          // than the HttpServer
                                                          // idle timeout value.

    // HTTP Request/Response header items
    private static final String ACCEPT = "Accept:";
    private static final String CACHE_CONTROL = "Cache-control:";
    private static final String CONTENT_LENGTH = "Content-Length:";
    private static final String CONTENT_TYPE = "Content-Type:";
    private static final String CONNECTION = "Connection:";
    private static final String HOST = "Host:";
    private static final String TRANSFER_ENCODING = "Transfer-Encoding:";
    // End-of-line for HTTP header items
    private static final String EOL = "\r\n";

    // Some common content types
    public static final String CONTENT_FORM_URLENCODED = "application/x-www-form-urlencoded";

    // The Default character encoding for HTTP
    private static final String DEFAULT_CHAR_ENCODING = "ISO-8859-1";

    /*
     * Minimum allowed value for idle timeout, which is how long a socket can
     * remain idle before it is closed.
     */
    final static int MIN_IDLE_TIMEOUT = 0;

    // The configuration values used at run time
    private static int REQUEST_TIMEOUT;
    private static int REQUEST_RETRY_COUNT;
    private static volatile int IDLE_TIMEOUT;

    // The set of providers for various supported protocols.
    // This is populated from global.properties from a set of entries of
    // the form
    // http.client.provider.<protocol> = className
    // By default, this class is registered for the http protocol, although
    // that can be overridden by an entry in the global.properties file.
    // The specified className must meet two criteria:
    // - it must be accessible
    // - it must have a public static factory method called getInstance that
    // returns an object that is a subclass of this class.
    // The map is keyed by protocol and the entries are providers.
    private final static Map s_providers = new HashMap ();

    // A flag to indicated if the config properties have
    // been read and initialized
    private static volatile boolean s_init = false;

    /**
     * Initialize the static constants by retrieving the configuration
     * parameters and also the providers. Once the properties are read and
     * initialized the s_init flag is set so that it is only done once. This was
     * moved from the static initializer because on a HttpClient.shutdown() we
     * don't need to read the props to shutdown when call from the
     * CoreWebAppShutdownListener in Tomcat.
     * 
     */
    private static synchronized void init ()
    {
        if (!s_init)
        {
            // Now the providers
            // Start with this class as the provider for http. Do this
            // first in case it is overridden in the properties file
            String myName = HttpClient.class.getName ();
            try
            {
                s_providers.put ("http", new Provider (myName));
            }
            catch (Exception e)
            {
                // This is very serious - this class isn't a valid provider!
                logger.warn ("Couldn't register " + myName + " as default http client provider.", e);
            }

            PfyProperties props = null;
            try
            {
                props = PfyPropertiesFactory.getProperties ("global");
            }
            catch (ConfigException e1)
            {
                logger.warn (e1.getMessage ());
            }

            // First the simple values
            REQUEST_TIMEOUT = getPropValue (props, "http.client.requestTimeout", 5000, DEFAULT_REQUEST_TIMEOUT);
            REQUEST_RETRY_COUNT = getPropValue (props, "http.client.retryCount", 0, DEFAULT_REQUEST_RETRY_COUNT);
            IDLE_TIMEOUT = getPropValue (props, "http.client.idleTimeout", MIN_IDLE_TIMEOUT, DEFAULT_IDLE_TIMEOUT);

            // Now all the other providers in the properties file
            Properties providers = props.getSubProperties ("http.client.provider.");

            if (providers.size () > 0)
            {
                for (Enumeration ee = providers.keys (); ee.hasMoreElements ();)
                {
                    String proto = (String) ee.nextElement ();
                    String className = providers.getProperty (proto);
                    if (proto == null)
                    {
                        // This is impossible, we just got the key from
                        // the map, there has to be a value. But just in case
                        logger.warn ("Couldn't find provider for protocol " + proto);
                        continue;
                    }
                    try
                    {
                        s_providers.put (proto.toLowerCase (), new Provider (className));
                    }
                    catch (Exception e)
                    {
                        logger.warn ("Couldn't register " + className + " as http client provider for " + proto);
                    }
                }

                s_init = true;
            }
            if (logger.isTraceEnabled ())
            {
                StringBuffer buff = new StringBuffer ("HttpClient configuration values\n");
                buff.append ("requestTimeout: ").append (REQUEST_TIMEOUT).append ('\n').append ("retryCount: ")
                    .append (REQUEST_RETRY_COUNT).append ('\n').append ("idleTimeout: ").append (IDLE_TIMEOUT)
                    .append ('\n');
                for (Iterator it = s_providers.entrySet ().iterator (); it.hasNext ();)
                {
                    Map.Entry entry = (Map.Entry) it.next ();
                    buff.append ("provider for ").append (entry.getKey ()).append (": ")
                        .append (((Provider) entry.getValue ()).getClassName ()).append ('\n');
                }
                logger.trace (buff.toString ());
            }

        }
    }

    /**
     * Helper method to get an integer property value. Used by the static
     * initializer.
     * 
     * @param props The properties to get the value from
     * @param key The name of the property value to retrieve
     * @param minValue The minimum value of the property
     * @param defaultValue The default value of the property
     * @return The value of the property. If the property is not defined or is
     *         not a valid integer or is less than the minimum value, returns
     *         the default value.
     */
    private static int getPropValue (PfyProperties props, String key, int minValue, int defaultValue)
    {
        int value = defaultValue;
        String s = props.getProperty (key);
        try
        {
            value = Integer.parseInt (s);
            if (value < minValue)
            {
                logger.warn ("Value for " + key + " must be >= " + minValue + "; configured value of " + value
                             + " ignored.");
                value = defaultValue;
            }
        }
        catch (NumberFormatException fne)
        {
            // No message if it is missing, but if its there and invalid
            // tell someone about it.
            if (s != null)
            {
                logger.warn ("Value for " + key + " must be numeric; configured value of " + s + "ignored.");
            }
        }
        return value;
    }

    //
    // Instance variables
    //
    /** Protocol for this connection */
    protected String m_protocol = "http";// Connection protocol
    /** Server host name */
    protected String m_host; // The host to send the request
                             // to
    /** Server port */
    protected int m_port = 80; // The port to connect to
    /** Server resource to request */
    protected String m_resource; // The resource to request on
                                 // that host
    /** Query string in request */
    protected String m_query = ""; // query part of request
    /** Keep socket open between requests? */
    private boolean m_keepAlive = true; // Keep connection open?

    private Set m_requests = Collections.synchronizedSet (new HashSet ()); // Set
                                                                           // of
                                                                           // objects

    // that handle individual requests

    /**
     * Create an HttpClient to the specified host for the specified resource.
     * 
     * @param host A string specifying the host name
     * @param resource A string specifying the resource name.
     */
    private HttpClient (String host, String resource)
    {
        init ();
        setHostAndPort (host);
        m_resource = resource;
    }

    /**
     * Create an HttpClient to the specified host for the specified resource,
     * including an initial query string.
     * 
     * @param host A string specifying the host name
     * @param resource A string specifying the resource name.
     * @param query Query part of request (e.g. "?p1=v1&p2=v2")
     */
    private HttpClient (String host, String resource, String query)
    {
        init ();
        setHostAndPort (host);
        m_resource = resource;
        m_query = query;
    }

    /**
     * Create an HttpClient to the specified url.
     * 
     * @param url A string containing the entire url including protocol. The
     *            protocol must be http
     */
    private HttpClient (String url)
    {
        init ();
        // test for protocol
        if (!url.startsWith ("http://"))
        {
            throw new IllegalArgumentException ("Invalid protocol");
        }

        // finish the setup
        finishSetup (url.substring (7));
    }

    /**
     * Shuts down resources that are associated with HttpClient. This includes
     * closing idle connections, and shutting down threads that manage the
     * connection pools, etc.. This is exposed as a public method so that it can
     * be called by {@link CoreWebAppShutdownListener}.
     */
    public static void shutdown ()
    {
        ConnectionPool.shutdown ();
    }

    /**
     * Finish the initialization process. This method takes a string that has
     * the entire URL except for the protocol and completes the set up process.
     * For use by sub class constructors.
     * 
     * @param url the url of the resource, with the protocol (including the ://)
     *            stripped off.
     */
    protected final void finishSetup (String url)
    {
        String resource = "";

        // locate resource
        int p1 = url.indexOf ('/');
        if (p1 != -1)
        {
            resource = url.substring (p1);
            url = url.substring (0, p1);
        }

        // split out host and port
        setHostAndPort (url);

        // split out resource and query
        int p2 = resource.indexOf ('?');
        if (p2 != -1)
        {
            m_query = resource.substring (p2);
            resource = resource.substring (0, p2);
        }
        m_resource = resource;
    }

    /**
     * Create an HttpClient to the specified url.
     * 
     * @param url A URL containing the entire url including protocol. The
     *            protocol must be http
     */
    private HttpClient (URL url)
    {
        init ();
        m_protocol = url.getProtocol ();
        if (!"http".equals (m_protocol))
        {
            throw new IllegalArgumentException ("Invalid protocol");
        }

        finishSetup (url);
    }

    /**
     * Finish the initialization process. This method ignores the protocol part
     * of the URL but processes the rest of it. For use by sub class
     * constructors.
     * 
     * @param url the URL of the resource.
     */
    protected final void finishSetup (URL url)
    {
        m_host = url.getHost ();
        if (url.getPort () != -1)
        {
            // Specified a port, use it instead of the default
            m_port = url.getPort ();
        }
        m_resource = HttpUtil.getPath (url.getFile ());
        m_query = HttpUtil.getQuery (url.getFile ());
        if (m_query != null)
        {
            m_query = "?" + m_query;
        }
        else
        {
            m_query = "";
        }
    }

    /**
     * Default constructor for use by derived classes. Classes using this
     * constructor must call one of the finishSetup() methods before using the
     * class.
     */
    protected HttpClient ()
    {
        init ();
    }

    /**
     * Sets the time that a socket can remain idle before it is closed and
     * removed from the pool.
     * 
     * @param idleTimeout The idle time in msecs.
     */
    static void setIdleTimeout (int idleTimeout)
    {
        if (idleTimeout < MIN_IDLE_TIMEOUT)
        {
            throw new IllegalArgumentException ("idleTimeout must be greater than or equal to " + MIN_IDLE_TIMEOUT
                                                + '.');
        }
        IDLE_TIMEOUT = idleTimeout;
    }

    /**
     * Gets the time that sockets can remain idle before being closed and
     * removed from the pool.
     * 
     * @return The idle time in msecs.
     */
    static int getIdleTimeout ()
    {
        return IDLE_TIMEOUT;
    }

    /**
     * Gets the number of sockets in the idle pool.
     * 
     * @return the number of sockets.
     */
    static int getIdlePoolSize ()
    {
        return ConnectionPool.getIdlePoolSize ();
    }

    /**
     * Closes all sockets in the idle pool, and removes them from the pool.
     */
    static void emptyIdlePool ()
    {
        ConnectionPool.emptyIdlePool ();
    }

    /**
     * Create an HttpClient based on a URL.
     * 
     * @param url URL defining the server to connect to.
     * @return An <code>HttpClient</code> instance for the specified URL.
     * @throws IllegalArgumentException if there is no provider for the protocol
     *             specified in the URL or if the constructor for the provider
     *             throws an exception.
     * @throws ProviderException if there is an error in invoking the provider's
     *             constructor.
     * @see #getHttpClient(String)
     * @see #getExclusiveHttpClient(String)
     */
    public static HttpClient getHttpClient (URL url)
    {
        return createInstance (url.toString ());
    }

    /**
     * Create an HttpClient based on the String containing a URL.
     * 
     * @param url A String containing the entire url including protocol.
     * @return An <code>HttpClient</code> instance for the specified URL.
     * @throws IllegalArgumentException if there is no provider for the protocol
     *             specified in the URL or if the constructor for the provider
     *             throws an exception.
     * @throws ProviderException if there is an error in invoking the provider's
     *             constructor.
     * @see #getHttpClient(URL)
     * @see #getExclusiveHttpClient(String)
     */
    public static HttpClient getHttpClient (String url)
    {
        return createInstance (url);
    }

    /**
     * Create an exclusive HttpClient based on the protocol. This instance is
     * not cached.
     * 
     * @param url A String containing the entire url including protocol
     * @return An <code>HttpClient</code> instance for the specified URL.
     *         Beginning with Core 1.2, there is no difference between this call
     *         and getHttpClient.
     * @throws IllegalArgumentException if there is no provider for the protocol
     *             specified in the URL or if the constructor for the provider
     *             throws an exception.
     * @throws ProviderException if there is an error in invoking the provider's
     *             constructor.
     * @see #getHttpClient(URL)
     * @see #getHttpClient(String)
     * @deprecated Use @link{getHttpClient(URL)}getHttpClient
     */
    public static HttpClient getExclusiveHttpClient (String url)
    {
        return getHttpClient (url);
    }

    /**
     * Create an HttpClient based on the String containing a URL. This is public
     * as an implementation side effect. It should not be called by client
     * applications.
     * 
     * @param url A String containing the entire url including protocol.
     * @return An <code>HttpClient</code> instance for the specified URL.
     * @throws IllegalArgumentException if there is no provider for the protocol
     *             specified in the URL or if the constructor for the provider
     *             throws an exception.
     * @throws ProviderException if there is an error in invoking the provider's
     *             constructor.
     * @see #getHttpClient(URL)
     * @see #getExclusiveHttpClient(String)
     */
    public static HttpClient getInstance (String url)
    {
        return new HttpClient (url);
    }

    /*
     * getInstance does the work of creating the HttpClient based on url.
     */
    private static HttpClient createInstance (String url)
    {
        init ();

        // Find the protocol, then instantiate a new
        // instance of the appropriate provider.
        int colon = url.indexOf (':');
        if (colon == -1)
        {
            throw new IllegalArgumentException ("No protocol specified in URL: " + url);
        }
        String proto = url.substring (0, colon).toLowerCase ();
        Provider provider = (Provider) s_providers.get (proto);
        if (provider == null)
        {
            throw new IllegalArgumentException ("No provider for protocol " + proto + " in URL " + url);
        }
        if (logger.isDebugEnabled ())
        {
            logger.trace ("Creating instance of " + provider.getClassName () + " for URL " + url);
        }
        return provider.getInstance (url);
    }

    /**
     * Return string form of HttpClient
     * 
     * @return The URL to which this client is connected.
     */
    public String toString ()
    {
        return m_protocol + "://" + m_host + ":" + m_port + m_resource;
    }

    /**
     * Get the host to which this HttpClient is connected.
     * 
     * @return The host to which this instance is connected.
     */
    public String getHost ()
    {
        return m_host;
    }

    /**
     * Get the port to which this HttpClient is connected.
     * 
     * @return The port number to which this instance is connected
     */
    public int getPort ()
    {
        return m_port;
    }

    /**
     * Set the host and port.
     * 
     * Splits a host string into host and port and sets these values in the
     * member variables m_host and m_port. The host parameter can be either an
     * IPV4 host and port in the format 192.168.1.100:3890 or hostname:3890. Or
     * it can be in IPV6 format such as [fe80::248a:f9a0:268a:e3c1]:3071. For
     * IPV6 the [] are required around the host portion.
     * 
     * @param host String containing the host and port
     * 
     */
    protected void setHostAndPort (String host)
    {
        if (host.indexOf ('[') >= 0)
        {
            int p = host.indexOf ('[');
            int p1 = host.indexOf (']');

            if (p1 == -1 || p1 <= p)
            {
                throw new IllegalArgumentException ("Invalid IPV6 address");
            }

            m_host = host.substring (p, p1 + 1);

            p = host.indexOf (':', p1);
            if (p != -1)
            {
                String port = host.substring (p + 1, host.length ());
                m_port = Integer.parseInt (port);
            }
        }
        else
        {
            int p = host.indexOf (':');
            if (p != -1)
            {
                m_host = host.substring (0, p);
                String port = host.substring (p + 1, host.length ());
                m_port = Integer.parseInt (port);
            }
            else
            {
                m_host = host;
            }
        }
    }

    /**
     * getMyHost
     * 
     * Opens a socket and discovers the client-side ip address and name that
     * will be used with the connection.
     * 
     * @param getAddress get the ip address
     * @param getFQName get the fully qualified name
     * @return ipAddress or name
     */
    public String getMyHost (boolean getAddress, boolean getFQName) throws UnknownHostException, IOException,
                                                                   MSHttpException
    {
        Socket s = null;
        String retVal = null;

        try
        {
            s = getSocket ();
            InetAddress ia = s.getLocalAddress ();
            if (getAddress)
            {
                // return an address
                retVal = ia.getHostAddress ();
            }
            else
            {
                retVal = ia.getHostName ();

                // we want to return a name, make sure we
                // didn't get back an ip address
                String[] nParts = StringUtil.getArray (retVal, ".");
                if (nParts.length == 4 && StringUtil.isNumeric (nParts[0]) && StringUtil.isNumeric (nParts[1])
                    && StringUtil.isNumeric (nParts[2]) && StringUtil.isNumeric (nParts[3]))
                {
                    throw new UnknownHostException ();
                }

                if (!getFQName)
                {
                    // strip off the domain part
                    retVal = nParts[0];
                }
            }
        }
        finally
        {
            // close the socket we used
            if (s != null)
                s.close ();
        }

        return (retVal);
    }

    /**
     * getMyHostAddress
     * 
     * Returns my host address
     * 
     * @return ipAddress
     */
    public String getMyHostAddress () throws UnknownHostException, IOException, MSHttpException
    {
        return (getMyHost (true, false));
    }

    /**
     * getMyHostName
     * 
     * Returns my host address
     * 
     * @return name
     */
    public String getMyHostName () throws UnknownHostException, IOException, MSHttpException
    {
        return (getMyHost (false, false));
    }

    /**
     * getMyFullyQualifiedHostName
     * 
     * Returns my host address
     * 
     * @return fqName
     */
    public String getMyFullyQualifiedHostName () throws UnknownHostException, IOException, MSHttpException
    {
        return (getMyHost (false, true));
    }

    /**
     * Get the keep alive value.
     * 
     * @return <code>true</code> if the connection will be kept open between
     *         requests; else <code>false</code>
     * @see #setKeepAlive
     */
    public boolean getKeepAlive ()
    {
        return m_keepAlive;
    }

    /**
     * Set the keep alive value. If keep alive is true, then the socket
     * connection to the server will be kept open between requests, if this
     * option is supported by the server. If keep alive is false, then the
     * socket connection will be closed after each request.
     * 
     * @param keepAlive the keepAlive value
     */
    public void setKeepAlive (boolean keepAlive)
    {
        m_keepAlive = keepAlive;
    }

    /**
     * Closes all open requests. Any outstanding requests are terminated without
     * waiting for results to be returned.
     */
    public void close ()
    {
        synchronized (m_requests)
        {
            for (Iterator it = m_requests.iterator (); it.hasNext ();)
            {
                HttpClientRequest hcr = (HttpClientRequest) it.next ();
                hcr.closeRequest ();
            }
        }
    }

    /**
     * Sends a GET request and returns the response.
     * 
     * Default values are used for the timeout value and retry count. The query
     * part is assumed to have been included with the constructor call.
     * 
     * @return A string containing the body of the response
     * 
     * @throws MSHttpConnectionException if an error is encountered when trying
     *             to connect to the server.
     * @throws MSHttpTimeoutException if the response is not received within the
     *             time specified by the default timeout value
     * @throws MSHttpException If some other error (such as an IO Exception) is
     *             encountered. Note that MsHttpConnectionException and
     *             MSHttpTimeoutException are subclasses of MSHttpException. If
     *             the caller doesn't wish to distinguish among the exceptions,
     *             a single catch clause for MSHttpException is all that is
     *             necessary. If the caller does wish to distinguish among the
     *             exceptions, the catch clause for MSHttpConnectionException
     *             and MSHttpTimeoutException must come before the catch clause
     *             for MSHttpException
     */
    public String get () throws MSHttpConnectionException, MSHttpTimeoutException, MSHttpException
    {
        // A convenience function - just call the real one
        return get (null, REQUEST_TIMEOUT, REQUEST_RETRY_COUNT);
    }

    /**
     * Sends a GET request and returns the response.
     * 
     * Default values are used for the timeout value and retry count.
     * 
     * @param query Query part of request (e.g. "?p1=v1&p2=v2")
     * @return A string containing the body of the response
     * 
     * @throws MSHttpConnectionException if an error is encountered when trying
     *             to connect to the server.
     * @throws MSHttpTimeoutException if the response is not received within the
     *             time specified by the default timeout value
     * @throws MSHttpException If some other error (such as an IO Exception) is
     *             encountered. Note that MsHttpConnectionException and
     *             MSHttpTimeoutException are subclasses of MSHttpException. If
     *             the caller doesn't wish to distinguish among the exceptions,
     *             a single catch clause for MSHttpException is all that is
     *             necessary. If the caller does wish to distinguish among the
     *             exceptions, the catch clause for MSHttpConnectionException
     *             and MSHttpTimeoutException must come before the catch clause
     *             for MSHttpException
     */
    public String get (String query) throws MSHttpConnectionException, MSHttpTimeoutException, MSHttpException
    {
        // A convenience function - just call the real one
        return get (query, REQUEST_TIMEOUT, REQUEST_RETRY_COUNT);
    }

    /**
     * Sends a GET request and returns the response.
     * 
     * The query, timeout and retry count are specified as arguments.
     * 
     * @param query Query part of request (e.g. "?p1=v1&p2=v2")
     * @param timeout The length of time to wait for a response, in msec
     * @param retryCount The number of times to retry if a response isn't
     *            received.
     * @return A string containing the body of the response
     * 
     * @throws MSHttpConnectionException if an error is encountered when trying
     *             to connect to the server.
     * @throws MSHttpTimeoutException if the response is not received within the
     *             time specified by the timeout parameter
     * @throws MSHttpException If some other error (such as an IO Exception) is
     *             encountered. Note that MsHttpConnectionException and
     *             MSHttpTimeoutException are subclasses of MSHttpException. If
     *             the caller doesn't wish to distinguish among the exceptions,
     *             a single catch clause for MSHttpException is all that is
     *             necessary. If the caller does wish to distinguish among the
     *             exceptions, the catch clause for MSHttpConnectionException
     *             and MSHttpTimeoutException must come before the catch clause
     *             for MSHttpException
     */
    public String get (String query, int timeout, int retryCount) throws MSHttpConnectionException,
                                                                 MSHttpTimeoutException, MSHttpException
    {
        // The real implementation
        String fullQuery = m_query;
        if (query != null)
        {
            // If there is a leading ? or &, get rid of it. We'll put the
            // appropriate one back later
            if (query.startsWith ("?") || query.startsWith ("&"))
            {
                query = query.substring (1);
            }

            // Need to combine this query with the query from the initial
            // URL
            if (fullQuery == null || fullQuery.length () == 0)
            {
                // No query in initial URL, just use this one
                fullQuery = '?' + query;
            }
            else
            {
                // Combine this query with the one initially specified
                fullQuery += '&' + query;
            }
        }

        // Create an object to process the request.
        // Need to keep track of these in case the close
        // method is called and we need to shut it down
        HttpClientRequest req = new HttpGetRequest (fullQuery, m_keepAlive);
        m_requests.add (req);
        try
        {
            return req.getResponse (timeout, retryCount);
        }
        finally
        {
            m_requests.remove (req);
        }
    }

    /**
     * Sends a POST request and returns the response.
     * 
     * Default values are used for the timeout value and retry count.
     * 
     * @param content A string to be included as the body of the POST request.
     *            This value is not encoded in any way. It is the responsibility
     *            of the requestor to do any necessary encoding before calling
     *            this method.
     * @param contentType The value of the Content-Type http header. If this is
     *            null, then no Content-Type information will be included in the
     *            request.
     * @return A string containing the body of the response
     * 
     * @throws MSHttpConnectionException if an error is encountered when trying
     *             to connect to the server.
     * @throws MSHttpTimeoutException if the response is not received within the
     *             time specified by the default timeout value
     * @throws MSHttpException If some other error (such as an IO Exception) is
     *             encountered. Note that MsHttpConnectionException and
     *             MSHttpTimeoutException are subclasses of MSHttpException. If
     *             the caller doesn't wish to distinguish among the exceptions,
     *             a single catch clause for MSHttpException is all that is
     *             necessary. If the caller does wish to distinguish among the
     *             exceptions, the catch clause for MSHttpConnectionException
     *             and MSHttpTimeoutException must come before the catch clause
     *             for MSHttpException
     * @throws IllegalArgumentException if <code>content</code> is
     *             <code>null</code>
     * @see #post(String, String, int, int)
     * @see #post(byte[], String)
     */
    public String post (String content, String contentType) throws MSHttpConnectionException, MSHttpTimeoutException,
                                                           MSHttpException
    {
        // A convenience function - just call the real one
        return post (content, contentType, REQUEST_TIMEOUT, REQUEST_RETRY_COUNT);
    }

    /**
     * Sends a POST request and returns the response.
     * 
     * The timeout and retry count are specified as arguments.
     * 
     * @param content A string to be included as the body of the POST request.
     *            This value is not encoded in any way. It is the responsibility
     *            of the requestor to do any necessary encoding before calling
     *            this method.
     * @param contentType The value of the Content-Type http header. If this is
     *            null, then no Content-Type information will be included in the
     *            request.
     * @param timeout The length of time to wait for a response, in msec
     * @param retryCount The number of times to retry if a response isn't
     *            received.
     * @return A string containing the body of the response
     * 
     * @throws MSHttpConnectionException if an error is encountered when trying
     *             to connect to the server.
     * @throws MSHttpTimeoutException if the response is not received within the
     *             time specified by the timeout parameter
     * @throws MSHttpException If some other error (such as an IO Exception) is
     *             encountered. Note that MsHttpConnectionException and
     *             MSHttpTimeoutException are subclasses of MSHttpException. If
     *             the caller doesn't wish to distinguish among the exceptions,
     *             a single catch clause for MSHttpException is all that is
     *             necessary. If the caller does wish to distinguish among the
     *             exceptions, the catch clause for MSHttpConnectionException
     *             and MSHttpTimeoutException must come before the catch clause
     *             for MSHttpException.
     * @throws IllegalArgumentException if <code>content</code> is
     *             <code>null</code>
     * @see #post(byte[], String, int, int)
     */
    public String post (String content, String contentType, int timeout, int retryCount)
                                                                                        throws MSHttpConnectionException,
                                                                                        MSHttpTimeoutException,
                                                                                        MSHttpException
    {
        if (content == null)
        {
            throw new IllegalArgumentException ("Must specify content for POST request");
        }
        try
        {
            return post (content.getBytes (DEFAULT_CHAR_ENCODING), contentType, timeout, retryCount);
        }
        catch (UnsupportedEncodingException e)
        {
            // This should never happen. The Java spec requires that all
            // implementations
            // support the ISO-8859-1 encoding.
            logger.warn ("Unsupported Encoding Exception for required encoding", e);
            return "";
        }
    }

    /**
     * Sends a POST request and returns the response.
     * 
     * Default values are used for the timeout value and retry count.
     * 
     * @param content A <code>Map</code> containing field names and values. Each
     *            entry in the map must have a key that is a String and and
     *            value that is either a String or String array. The entries are
     *            converted to a URL encoded string and sent as the body of the
     *            request. This simulates what would be done by an Html FORM
     *            element.
     * @return A string containing the body of the response
     * 
     * @throws MSHttpConnectionException if an error is encountered when trying
     *             to connect to the server.
     * @throws MSHttpTimeoutException if the response is not received within the
     *             time specified by the timeout parameter
     * @throws MSHttpException If some other error (such as an IO Exception) is
     *             encountered. Note that MsHttpConnectionException and
     *             MSHttpTimeoutException are subclasses of MSHttpException. If
     *             the caller doesn't wish to distinguish among the exceptions,
     *             a single catch clause for MSHttpException is all that is
     *             necessary. If the caller does wish to distinguish among the
     *             exceptions, the catch clause for MSHttpConnectionException
     *             and MSHttpTimeoutException must come before the catch clause
     *             for MSHttpException.
     * @throws IllegalArgumentException if <code>content</code> is
     *             <code>null</code> or any of the keys or values in the Map are
     *             not Strings.
     * @see #post(String,String)
     * @see #post(Map, int, int)
     */
    public String post (Map content) throws MSHttpConnectionException, MSHttpTimeoutException, MSHttpException
    {
        return post (content, REQUEST_TIMEOUT, REQUEST_RETRY_COUNT);
    }

    /**
     * Sends a POST request and returns the response.
     * 
     * Default values are used for the timeout value and retry count.
     * 
     * @param content A <code>Map</code> containing field names and values. Each
     *            entry in the map must have a key that is a String and and
     *            value that is either a String or String array. The entries are
     *            converted to a URL encoded string and sent as the body of the
     *            request. This simulates what would be done by an Html FORM
     *            element.
     * @return A string containing the body of the response
     * 
     * @throws MSHttpConnectionException if an error is encountered when trying
     *             to connect to the server.
     * @throws MSHttpTimeoutException if the response is not received within the
     *             time specified by the timeout parameter
     * @throws MSHttpException If some other error (such as an IO Exception) is
     *             encountered. Note that MsHttpConnectionException and
     *             MSHttpTimeoutException are subclasses of MSHttpException. If
     *             the caller doesn't wish to distinguish among the exceptions,
     *             a single catch clause for MSHttpException is all that is
     *             necessary. If the caller does wish to distinguish among the
     *             exceptions, the catch clause for MSHttpConnectionException
     *             and MSHttpTimeoutException must come before the catch clause
     *             for MSHttpException.
     * @throws IllegalArgumentException if <code>content</code> is
     *             <code>null</code> or any of the keys or values in the Map are
     *             not Strings.
     * @see #post(String,String)
     * @see #post(Map, int, int)
     */
    public String post (Map content, int timeout, int retryCount) throws MSHttpConnectionException,
                                                                 MSHttpTimeoutException, MSHttpException
    {
        if (content == null)
        {
            throw new IllegalArgumentException ("Must specify content for POST request");
        }
        StringBuffer buff = new StringBuffer ();
        for (Iterator it = content.entrySet ().iterator (); it.hasNext ();)
        {
            Map.Entry entry = (Map.Entry) it.next ();
            Object key = entry.getKey ();
            Object value = entry.getValue ();
            if (!(key instanceof String))
            {
                throw new IllegalArgumentException ("All keys must be of type String");
            }
            if (buff.length () > 0)
                buff.append ('&');
            String encodedKey = HttpUtil.URLEncode (key.toString ());
            if (value == null)
            {
                buff.append (encodedKey).append ('=');
            }
            else if (value instanceof String)
            {
                buff.append (encodedKey).append ('=').append (HttpUtil.URLEncode (value.toString ()));
            }
            else if (value instanceof String[])
            {
                String[] valList = (String[]) value;
                if (valList.length == 0)
                {
                    buff.append (encodedKey).append ('=');
                }
                else
                {
                    for (int i = 0; i < valList.length; i++)
                    {
                        if (i > 0)
                            buff.append ('&');
                        buff.append (encodedKey).append ('=').append (HttpUtil.URLEncode (valList[i]));
                    }
                }
            }
            else
            {
                throw new IllegalArgumentException ("All values must be String or String[]");
            }
        }

        return post (buff.toString (), CONTENT_FORM_URLENCODED, timeout, retryCount);
    }

    /**
     * Sends a POST request and returns the response.
     * 
     * Default values are used for the timeout value and retry count.
     * 
     * @param content Binary data to be included as the body of the POST
     *            request. This value is not encoded in any way. If the binary
     *            data is to be encoded as a string (i.e. Base64) then the
     *            requestor should do the necessary encoding and call the
     *            {@link #post(String, String) post} method that takes a String
     *            argument.
     * @param contentType The value of the Content-Type http header. If this is
     *            null, then no Content-Type information will be included in the
     *            request.
     * @return A string containing the body of the response
     * 
     * @throws MSHttpConnectionException if an error is encountered when trying
     *             to connect to the server.
     * @throws MSHttpTimeoutException if the response is not received within the
     *             time specified by the timeout parameter
     * @throws MSHttpException If some other error (such as an IO Exception) is
     *             encountered. Note that MsHttpConnectionException and
     *             MSHttpTimeoutException are subclasses of MSHttpException. If
     *             the caller doesn't wish to distinguish among the exceptions,
     *             a single catch clause for MSHttpException is all that is
     *             necessary. If the caller does wish to distinguish among the
     *             exceptions, the catch clause for MSHttpConnectionException
     *             and MSHttpTimeoutException must come before the catch clause
     *             for MSHttpException.
     * @throws IllegalArgumentException if <code>content</code> is
     *             <code>null</code>
     * @see #post(String,String)
     * @see #post(byte[], String, int, int)
     */
    public String post (byte[] content, String contentType) throws MSHttpConnectionException, MSHttpTimeoutException,
                                                           MSHttpException
    {
        return post (content, contentType, REQUEST_TIMEOUT, REQUEST_RETRY_COUNT);
    }

    /**
     * Sends a POST request and returns the response.
     * 
     * The timeout and retry count are specified as arguments.
     * 
     * @param content Binary data to be included as the body of the POST
     *            request. This value is not encoded in any way. If the binary
     *            data is to be encoded as a string (i.e. Base64) then the
     *            requestor should do the necessary encoding and call the
     *            {@link #post(String, String, int, int) post} method that takes
     *            a String argument.
     * @param contentType The value of the Content-Type http header. If this is
     *            null, then no Content-Type information will be included in the
     *            request.
     * @param timeout The length of time to wait for a response, in msec
     * @param retryCount The number of times to retry if a response isn't
     *            received.
     * @return A string containing the body of the response
     * 
     * @throws MSHttpConnectionException if an error is encountered when trying
     *             to connect to the server.
     * @throws MSHttpTimeoutException if the response is not received within the
     *             time specified by the timeout parameter
     * @throws MSHttpException If some other error (such as an IO Exception) is
     *             encountered. Note that MsHttpConnectionException and
     *             MSHttpTimeoutException are subclasses of MSHttpException. If
     *             the caller doesn't wish to distinguish among the exceptions,
     *             a single catch clause for MSHttpException is all that is
     *             necessary. If the caller does wish to distinguish among the
     *             exceptions, the catch clause for MSHttpConnectionException
     *             and MSHttpTimeoutException must come before the catch clause
     *             for MSHttpException
     * @throws IllegalArgumentException if <code>content</code> is
     *             <code>null</code>
     * @see #post(byte[], String)
     * @see #post(String, String, int, int)
     */
    public String post (byte[] content, String contentType, int timeout, int retryCount)
                                                                                        throws MSHttpConnectionException,
                                                                                        MSHttpTimeoutException,
                                                                                        MSHttpException
    {
        // Create an object to process the request.
        // Need to keep track of these in case the close
        // method is called and we need to shut it down
        HttpClientRequest req = new HttpPostRequest (content, contentType, m_keepAlive);
        m_requests.add (req);
        try
        {
            return req.getResponse (timeout, retryCount);
        }
        finally
        {
            m_requests.remove (req);
        }
    }

    /** Get a socket connection to the server. */
    protected Socket getSocket () throws UnknownHostException, IOException, MSHttpException
    {
        // Get a socket from the connection pool
        // If there isn't one in the pool, then create one
        StringBuffer buff = new StringBuffer ();
        Socket sock = ConnectionPool.getConnection (m_host, m_port);
        if (sock == null)
        {
            buff.append ("Creating new");
            sock = new Socket (m_host, m_port);
        }
        else
        {
            buff.append ("Using old");
        }
        if (logger.isDebugEnabled ())
        {
            buff.append (" socket connection to [").append (sock.toString ()).append (" ] to ").append (m_host)
                .append (':').append (m_port);
            logger.trace (buff.toString ());
        }
        return sock;
    }

    /**
     * Release a socket connection.
     * 
     * This implementation places the connection in a pool so that it can be
     * re-used if another request is sent to the same server. Classes that
     * derive from this class and have sockets that can't be re-used (e.g. SSL
     * sockets) should just close the socket
     */
    protected void releaseSocket (Socket sock)
    {
        if (sock != null)
        {
            if (IDLE_TIMEOUT != 0)
            {
                ConnectionPool.addConnection (m_host, m_port, sock);
            }
            else
            {
                // Don't use pooling
                if (logger.isDebugEnabled ())
                {
                    logger.debug ("Closing socket connection to " + m_host + ':' + m_port + '.');
                }
                try
                {
                    sock.close ();
                }
                catch (IOException ignored)
                {
                }
            }

        }
    }

    /**
     * Inner class to help with getting the response. An instance of this class
     * is created for each request made to the HttpClient instance. Subclasses
     * of this class should be defined for each of the types of http requests
     * supported. The subclass must implement the writeRequest method to send
     * the appropriate type of request.
     */
    private abstract class HttpClientRequest
    {
        private volatile int m_retryCount; // how many retries to do
        private int m_retries; // how many retries have been done
        private Socket m_conn = null; // socket connection to server
        protected boolean m_reqKeepAlive; // should socket be kept open after
                                          // request?
        private volatile boolean m_timedOut = false; // did this request time
                                                     // out?

        // Response codes we care about
        private static final String RESPONSE_CONTINUE = "100";
        private static final String RESPONSE_OK = "200";

        private HttpClientRequest (boolean keepAlive)
        {
            m_reqKeepAlive = keepAlive;
        }

        // Method called from HttpClient to get the response to
        // a request.
        private String getResponse (int timeout, int retryCount) throws MSHttpException
        {
            // Set up the environment and call tryResponse to actually
            // do the work
            m_retries = 0;
            m_retryCount = retryCount;
            String response = null;

            // get the response
            try
            {
                response = tryResponse (timeout);
            }
            catch (Exception e)
            {
                if (e instanceof MSHttpException)
                {
                    // A "normal" exception - just rethrow it
                    throw (MSHttpException) e;
                }
                else if (e instanceof Throwable)
                {
                    // Something unusual - wrap it and throw it
                    throw new MSHttpException (e, "Unexpected exception");
                }
            }
            finally
            {
                // Reset the logger
            }
            return (response);
        }

        /**
         * Closes the request. This is called from the HttpClient.close() method
         * to abort any open requests.
         * 
         */
        private void closeRequest ()
        {
            // set the possible number of retries to 0 and close things down
            m_retryCount = 0;
            closeSocket ();
        }

        /**
         * Closes socket connection and releases reader and writer
         * 
         */
        private synchronized void closeSocket ()
        {
            try
            {
                if (m_conn != null)
                {
                    logger.trace ("Closing socket connected to " + HttpClient.this.toString ());
                    m_conn.close ();
                }
            }
            catch (IOException ignored)
            {
            }
            finally
            {
                m_conn = null;
            }
        }

        private String tryResponse (int timeout) throws MSHttpException
        {
            // This method actually makes the http connection, sends the request
            // and gets the response.
            String response = "";
            BufferedReader in; // input reader on socket;
            DataOutputStream out; // output stream on socket;

            // If no retries are specified, we will still retry once
            // on any error except a timeout. Need to do this because
            // we can get "errors" that aren't really. For instance, if
            // we are keeping a socket connection open but the server's
            // timeout is shorter than ours, then we get an error writing
            // the request. Need to be able to recover from that.
            boolean retryOnError = (m_retryCount == 0);

            Exception excptn = null;
            for (m_retries = 0; m_retries < m_retryCount + 1; ++m_retries)
            {
                excptn = null;
                m_timedOut = false;
                // try block for creating connection
                try
                {
                    // open a socket connection
                    m_conn = getSocket ();
                    out = new DataOutputStream (new BufferedOutputStream (m_conn.getOutputStream ()));
                    in = new BufferedReader (new InputStreamReader (m_conn.getInputStream ()));
                }
                catch (IOException ex)
                {
                    closeSocket ();
                    logger.debug ("Exception creating connection", ex);
                    // Save the exception and try again
                    excptn = new MSHttpConnectionException (ex);
                    continue;
                }

                // try block to control disposing of the socket
                // the finally block associated with this try block
                // will either close the socket or return it to the
                // provider to be reused. The socket should not
                // be closed by any of the methods called from
                // within this try block, except possibly by the
                // timeout thread if the request is taking too
                // long to process.
                try
                {
                    // send the request
                    try
                    {
                        writeRequest (out);
                    }
                    catch (IOException ex)
                    {
                        logger.debug ("IO Exception while writing request", ex);
                        throw new MSHttpException (ex, "IO Exception while writing request");
                    }

                    // read the response
                    try
                    {
                        // Note that this may reset the keep alive value
                        response = readResponse (in, timeout);
                        // Got a response, can quit the retry loop
                        break;
                    }
                    catch (IOException ex)
                    {
                        if (m_timedOut)
                        {
                            logger.debug ("Timed out waiting for response");
                            throw new MSHttpTimeoutException ();
                        }
                        else
                        {
                            logger.debug ("IO Exception while reading response", ex);
                            throw new MSHttpException (ex, "IO Exception while reading response");
                        }
                    }
                }
                catch (MSHttpException ex)
                {
                    // A "normal" exception.
                    // Save the exception, mark the socket for
                    // closure, and continue.
                    excptn = ex;
                    m_reqKeepAlive = false;
                    continue;
                }
                catch (RuntimeException rte)
                {
                    // An unexpected exception.
                    // Mark the socket
                    // for closure, and rethrow the exception
                    m_reqKeepAlive = false;
                    throw rte;
                }
                finally
                {
                    if (m_reqKeepAlive)
                    {
                        // Can reuse socket
                        releaseSocket (m_conn);
                        m_conn = null;
                    }
                    else
                    {
                        // Getting rid of the socket
                        closeSocket ();
                    }
                    if (retryOnError && !m_timedOut)
                    {
                        retryOnError = false;
                        m_retries--;
                    }
                }
            } // end retry loop

            if (excptn != null)
            {
                // Went through the loop the maximum number of times, but
                // still had an excpetion
                if (!(excptn instanceof MSHttpException))
                {
                    // Wrap exception in MSHttpException
                    throw new MSHttpException (excptn, "Couldn't get http response");
                }
                else
                {
                    // Already an MSHttpException
                    throw (MSHttpException) excptn;
                }
            }

            if (logger.isTraceEnabled ())
            {
                String intro = "Response: ";
                String decodedResponse;
                try
                {
                    decodedResponse = HttpUtil.URLDecode (response);
                }
                catch (Exception e)
                {
                    // An error decoding the response
                    intro = "Response (not decoded): ";
                    decodedResponse = response;
                }
                logger.trace (intro + decodedResponse);
            }

            return response;
        }

        // Helper method that sends the request to the server
        protected abstract void writeRequest (DataOutputStream out) throws IOException;

        // Helper method that reads the response from the server.
        // Note that if there is a maximum time to wait for the
        // response, this method starts a timer thread. If the
        // time expires before the request has been received, the
        // timer thread will abort the read by closing the socket.
        // Regardless of whether the time is exceeded or not, the
        // timer thread is cleaned up before this method returns.
        // (i.e. - the lifetime of the timer thread is completely
        // contained within the lifetime of this method, regardless
        // of whether the request times out or not.)
        private String readResponse (BufferedReader in, int timeout) throws IOException, MSHttpException
        {
            Thread timerThread = null;
            String response = null;

            try
            {
                // start a timer thread if timeout is specified
                if (timeout > 0)
                {
                    timerThread = new TimerThread (timeout);
                    timerThread.start ();
                }

                String contentLength = null;
                String transferEncoding = null;
                logger.trace ("Waiting for response");

                // read response header from server
                ResponseHeader respHdr = readResponseHeader (in);

                // Did the server say to close the connection?
                String connStatus = respHdr.getItem (CONNECTION);
                if ("close".equalsIgnoreCase (connStatus))
                {
                    m_reqKeepAlive = false;
                }

                // Look for the header items we are interested in
                contentLength = respHdr.getItem (CONTENT_LENGTH);
                transferEncoding = respHdr.getItem (TRANSFER_ENCODING);

                if (contentLength == null && transferEncoding != null)
                {
                    // Tomcat jsp response is transferEncoding=chunked
                    if (transferEncoding.equalsIgnoreCase ("chunked"))
                    {
                        response = readChunkedResponse (in);
                    }
                    else
                    {
                        // An unsupported transfer encoding
                        throw new MSHttpException ("Unknown transfer encoding: " + transferEncoding);
                    }
                }
                else if (contentLength != null)
                {
                    // Most servers use content length
                    response = readSizedResponse (in, contentLength);
                }
                else
                {
                    // No content length or header, just read to EOF
                    response = readUnsizedResponse (in);

                    // Can't do keep alive here, since the only way
                    // to find the end was when the server closed the
                    // socket
                    m_reqKeepAlive = false;
                }
            }
            finally
            {
                if (timerThread != null)
                {
                    timerThread.interrupt ();
                    while (timerThread != null)
                    {
                        try
                        {
                            timerThread.join ();
                            timerThread = null;
                        }
                        catch (InterruptedException ignored)
                        {
                        }
                    }
                }
            }

            return response;
        }

        // Helper method that reads the HttpResponse header.
        private ResponseHeader readResponseHeader (BufferedReader in) throws IOException, MSHttpException
        {
            ResponseHeader hdr = new ResponseHeader ();
            String response = in.readLine ();
            if (response == null)
            {
                // Probably means that the socket connection with the
                // server is no longer any good. There is a time period
                // of several seconds after the server closes the socket
                // when we can still write on it and read from it
                // without getting an error. But reading returns EOF.
                // We'll just try again
                throw new MSHttpException ("Response header line was null");
            }
            if (response.length () == 0)
            {
                // This means the server doesn't follow the http protocol
                String msg = "Empty response status line";
                logger.fatal (msg);
                throw new MSHttpException (msg);
            }
            logger.trace ("Response status line: " + response);
            StringTokenizer tokenizer = new StringTokenizer (response, " ");
            if (tokenizer.countTokens () < 2)
            {
                String msg = "Invalid response status line: " + response;
                logger.fatal (msg);
                throw new MSHttpException (msg);
            }
            // The first token is the protocol - skip it
            // Next is the status
            tokenizer.nextToken ();
            String status = tokenizer.nextToken ();

            // The only statuses we can deal with are OK and Continue
            // Anything else is an error
            if (!status.equals (RESPONSE_OK) && !status.equals (RESPONSE_CONTINUE))
            {
                throw new MSHttpException ("Server error code: " + status);
            }
            hdr.setStatusCode (status);

            // The rest of the header is attribute-value pairs,
            // read them, looking for values that are of interest
            while ((response = in.readLine ()) != null)
            {
                logger.debug ("Response line: " + response);

                // get tokens separated by spaces
                tokenizer = new StringTokenizer (response, " ");

                // if no tokens, then this was a blank line
                // signifying the end of the header
                if (tokenizer.countTokens () == 0)
                    break;

                if (tokenizer.countTokens () < 2)
                {
                    // This shouldn't be possible
                    logger.fatal ("Unexpected format in response header line: " + response);
                    continue;
                }
                // Save the header value
                String attr = tokenizer.nextToken ();
                String value = tokenizer.nextToken ();
                hdr.addItem (attr, value);
            }

            if (hdr.getStatusCode ().equals (RESPONSE_CONTINUE))
            {
                // This was a continue response. We don't care about
                // it. It will be followed by the real response
                hdr = readResponseHeader (in);
            }
            return hdr;
        }

        // Helper method to read the response body if the header contained
        // an explicit content length.
        private String readSizedResponse (BufferedReader in, String contentLength) throws IOException
        {
            int cnt = Integer.parseInt (contentLength.trim ());

            // read until we get the number of characters specified
            // by the content size
            char[] chars = new char[cnt];
            int charsRead = 0;
            while (charsRead != cnt)
            {
                charsRead += in.read (chars, charsRead, cnt - charsRead);
            }
            return new String (chars);
        }

        // Helper method to read the response body if the header did not
        // contain an explicit content length. It reads to EOF.
        private String readUnsizedResponse (BufferedReader in) throws IOException
        {
            StringBuffer sb = new StringBuffer ();
            char[] bytes = new char[1024];
            int bytesRead = 0;
            while ((bytesRead = in.read (bytes, 0, 1024)) != -1)
            {
                sb.append (bytes, 0, bytesRead);
            }
            return sb.toString ();
        }

        // Helper method to read the response body if the header
        // indicated that it was sent using a chunked transfer
        // encoding.
        private String readChunkedResponse (BufferedReader in) throws IOException
        {
            StringBuffer sb = new StringBuffer ();
            String chunkSize;
            chunkSize = in.readLine (); // get the size
            if (chunkSize != null)
            {
                chunkSize = chunkSize.trim ();
            }
            else
            {
                chunkSize = "0";
            }
            logger.debug ("chunk-size: " + chunkSize);

            char[] bytes = null;
            int bytesLength = -1;
            // chunk size is specified in hex
            int length = Integer.parseInt (chunkSize, 16);
            while (length > 0)
            {
                int bytesRead = 0;
                if (length > bytesLength)
                {
                    bytes = new char[length];
                    bytesLength = length;
                }
                while (bytesRead < length)
                {
                    bytesRead += in.read (bytes, bytesRead, length - bytesRead);
                }

                String chunkData = new String (bytes, 0, length);
                logger.debug ("chunk-data: " + chunkData);
                sb.append (chunkData);

                // read next chunk size
                in.readLine ();
                chunkSize = in.readLine ();
                if (chunkSize != null)
                {
                    chunkSize = chunkSize.trim ();
                }
                else
                {
                    chunkSize = "0";
                }
                logger.debug ("chunk-size: " + chunkSize);
                length = Integer.parseInt (chunkSize, 16);
            }
            String footer = in.readLine ();
            logger.debug ("chunk-footer: " + footer);

            return sb.toString ();
        }

        /** Class that encapsulates a response header */
        private class ResponseHeader
        {
            private String m_statusCode;
            private Map m_headerItems = new HashMap ();

            void setStatusCode (String status)
            {
                m_statusCode = status;
            }

            String getStatusCode ()
            {
                return m_statusCode;
            }

            void addItem (String name, String value)
            {
                m_headerItems.put (name.toUpperCase (), value);
            }

            String getItem (String name)
            {
                return (String) m_headerItems.get (name.toUpperCase ());
            }
        } // End of inner class ResponseHeader
          // (net.pinfly.HttpClient.HttpClientRequest.ResponseHeader)

        /**
         * Thread that monitors timeout while waiting for a response.
         */
        private class TimerThread extends Thread
        {
            private final int m_timeToWait;

            private TimerThread (int timeToWait)
            {
                super ("TimerThread from " + Thread.currentThread ().getName ());
                m_timeToWait = timeToWait;
            }

            public void run ()
            {
                try
                {
                    logger.trace ("starting");
                    sleep (m_timeToWait);
                    // We woke up without getting interrupted
                    m_timedOut = true;
                    logger.trace ("time expired closing socket");
                    closeSocket ();
                }
                catch (InterruptedException ex)
                {
                    // We were interrupted, means we got a result.
                }
            }
        } // end of inner class TimerThread
          // (net.pinfly.HttpClient.HttpClientRequest.TimerThread)

    } // end of inner class HttpClientRequest
      // (net.pinfly.HttpClient.HttpClientRequest)

    /** A class that extends HttpClientRequest for handling GET requests */
    private class HttpGetRequest extends HttpClientRequest
    {
        private final String m_fullQuery;

        private HttpGetRequest (String fullQuery, boolean keepAlive)
        {
            super (keepAlive);
            m_fullQuery = fullQuery == null ? "" : fullQuery;
        }

        protected void writeRequest (DataOutputStream out) throws IOException
        {
            if (logger.isDebugEnabled ())
            {
                logger.debug ("Sending GET " + m_resource + m_fullQuery);
            }
            StringBuffer buff = new StringBuffer (100);
            buff.append ("GET ").append (m_resource).append (m_fullQuery).append (" HTTP/1.1").append (EOL)
                .append (HOST).append (' ').append (m_host).append (':').append (String.valueOf (m_port)).append (EOL)
                .append (ACCEPT).append (' ').append (CONTENT_FORM_URLENCODED).append (EOL).append (CACHE_CONTROL)
                .append (" max-age=0, must-revalidate").append (EOL);
            if (!m_reqKeepAlive)
            {
                buff.append (CONNECTION).append (" close").append (EOL);
            }
            buff.append (EOL);
            out.writeBytes (buff.toString ());
            out.flush ();
        }
    }

    /** A class that extends HttpClientRequest for handling POST requests */
    private class HttpPostRequest extends HttpClientRequest
    {
        private final byte[] m_content;
        private final String m_contentType;

        private HttpPostRequest (byte[] content, String contentType, boolean keepAlive)
        {
            super (keepAlive);
            if (content == null)
            {
                throw new IllegalArgumentException ("Must specify content for POST request");
            }
            m_content = content;
            m_contentType = contentType;
        }

        protected void writeRequest (DataOutputStream out) throws IOException
        {
            if (logger.isDebugEnabled ())
            {
                logger.debug ("Sending POST " + m_resource);
            }

            StringBuffer buff = new StringBuffer (100);
            buff.append ("POST ").append (m_resource).append (" HTTP/1.1").append (EOL).append (HOST).append (' ')
                .append (m_host).append (':').append (String.valueOf (m_port)).append (EOL).append (ACCEPT)
                .append (' ').append (CONTENT_FORM_URLENCODED).append (EOL).append (CACHE_CONTROL)
                .append (" max-age=0, must-revalidate").append (EOL);
            if (!m_reqKeepAlive)
            {
                buff.append (CONNECTION).append (" close").append (EOL);
            }
            if (m_contentType != null)
            {
                buff.append (CONTENT_TYPE).append (' ').append (m_contentType).append (EOL);
            }
            buff.append (CONTENT_LENGTH).append (' ').append (String.valueOf (m_content.length)).append (EOL)
                .append (EOL);
            out.writeBytes (buff.toString ());
            out.write (m_content, 0, m_content.length);
            out.flush ();
        }
    }

    // Inner class that maintains a pool of connected sockets.
    // When a socket is added to the pool, its expiration date is
    // set. A thread periodically checks idle sockets to see if they
    // have expired. If expired, the socket is removed from the pool
    // and closed.
    private static class ConnectionPool
    {
        // Socket Close Pool Config
        private final static int CLOSEPOOL_MAX_THREADS = 20; // Maximum
                                                             // concurrent
                                                             // threads to use
                                                             // to close
                                                             // sockets.
        private final static int CLOSEPOOL_IDLE_TIMEOUT = 300000; // How long a
                                                                  // close
                                                                  // thread can
                                                                  // be idle
                                                                  // before it
                                                                  // is
                                                                  // shutdown.

        // A pool responsible for closing sockets
        private final static SocketClosePool s_closePool = new SocketClosePool (CLOSEPOOL_MAX_THREADS,
                                                                                CLOSEPOOL_IDLE_TIMEOUT);

        // The set of pooled connections.
        // The key to the map is the host and port number to which
        // the socket is connected.
        // The value is a list of Connection instances for sockets
        // that are connected to that host/port combination.
        private static Hashtable s_connections = new Hashtable ();

        private final static IdleTimeOutThread s_idleTimeoutThread = new IdleTimeOutThread ();
        static
        {
            // Start a thread responsible for checking for expired
            // idle connections
            s_idleTimeoutThread.setDaemon (true);
            s_idleTimeoutThread.start ();
        }

        /**
         * Shuts down the thread that manages the idle pool, and closes any idle
         * connections.
         */
        private static void shutdown ()
        {
            s_idleTimeoutThread.interrupt ();

            try
            {
                s_idleTimeoutThread.join ();
            }
            catch (InterruptedException ignored)
            {
            }

            // close any idle connections
            emptyIdlePool ();

            // shutdown the close pool
            s_closePool.shutDownPool ();
        }

        /**
         * Gets the size of the idle pool
         * 
         * @return the size
         */
        static int getIdlePoolSize ()
        {
            int count = 0;
            synchronized (s_connections)
            {
                for (Iterator iter = s_connections.values ().iterator (); iter.hasNext ();)
                {
                    List conns = (List) iter.next ();
                    count += conns.size ();
                }
            }

            return count;
        }

        /**
         * Closes all idle connections, and removes
         * 
         * them from the idle pool.
         */
        static void emptyIdlePool ()
        {
            int count = 0;
            synchronized (s_connections)
            {
                for (Iterator iter = s_connections.entrySet ().iterator (); iter.hasNext ();)
                {
                    Map.Entry entry = (Map.Entry) iter.next ();
                    List conns = (List) entry.getValue ();
                    for (Iterator iterator = conns.iterator (); iterator.hasNext ();)
                    {
                        Connection conn = (Connection) iterator.next ();
                        s_closePool.add (conn.getSocket ());
                        iterator.remove ();
                    }
                    iter.remove ();
                }
            }
        }

        // Takes a socket connected to the given host and port
        // and adds it to the connection pool
        static void addConnection (String host, int port, Socket socket)
        {
            // Get a new Connection for the socket
            Connection conn = new Connection (host, port, socket);
            synchronized (s_connections)
            {
                // Get the list of Connections for this host and port,
                // creating one if it doesn't already exist.
                List connections = (List) s_connections.get (conn.getKey ());
                if (connections == null)
                {
                    connections = new ArrayList ();
                    s_connections.put (conn.getKey (), connections);
                }
                // Add this Connection to the list
                connections.add (conn);
            }
        }

        // Return a socket from the pool if there is one
        // connected to the specified host and port.
        // Returns null if there is no such socket.
        static Socket getConnection (String host, int port) throws UnknownHostException, IOException, MSHttpException
        {
            Socket sock = null;
            ConnectionKey key = new ConnectionKey (host, port);
            synchronized (s_connections)
            {
                List connections = (List) s_connections.get (key);
                if (connections != null)
                {
                    if (connections.size () > 0)
                    {
                        Connection conn = (Connection) connections.remove (0);
                        sock = conn.getSocket ();
                        if (logger.isDebugEnabled ())
                        {
                            logger.debug ("Reusing socket connection to " + host + ':' + port);
                        }
                    }

                    // If that is the last socket to that host/port remove the
                    // list
                    // from the cache
                    if (connections.size () == 0)
                    {
                        s_connections.remove (key);
                    }
                }

            }
            return sock;
        }

        // Helper class that encapsulates a socket open to a given
        // host and port.
        private static class Connection
        {
            // The host/port value used to identify the socket
            private final ConnectionKey m_key;
            // The socket itself
            private final Socket m_socket;

            // When this Connection should expire.
            private final long m_expireTime = System.currentTimeMillis () + IDLE_TIMEOUT;

            Connection (String host, int port, Socket socket)
            {
                m_key = new ConnectionKey (host, port);
                m_socket = socket;
            }

            ConnectionKey getKey ()
            {
                return m_key;
            }

            Socket getSocket ()
            {
                return m_socket;
            }

            boolean isExpired ()
            {
                return (System.currentTimeMillis () > m_expireTime);
            }

            public String toString ()
            {
                StringBuffer buff = new StringBuffer ("*** Connection ***\n");
                buff.append ("key = ").append (m_key).append ("\nsocket = ").append (m_socket)
                    .append ("\nexpirationTime = ").append (new Date (m_expireTime));
                return buff.toString ();
            }

        } // end of inner class Connection
          // (net.pinfly.http.HttpClient.ConnectionPool.Connection)

        // Class used to implement a key based on the host name
        // and port number. It just overrieds equals and hashCode
        // to use both values.
        private static class ConnectionKey
        {
            private final String m_host;
            private final int m_port;

            ConnectionKey (String host, int port)
            {
                m_host = host;
                m_port = port;
            }

            public int hashCode ()
            {
                return m_host.hashCode () + m_port;
            }

            public boolean equals (Object o)
            {
                boolean eq = false;
                if (o instanceof ConnectionKey)
                {
                    ConnectionKey other = (ConnectionKey) o;
                    eq = (m_port == other.m_port && m_host.equals (other.m_host));
                }
                return eq;
            }
        } // end of inner class ConnectionKey
          // (net.pinfly.http.HttpClient.HttpClientRequest.ConnectionKey)

        /**
         * A thread that is responsible for periodically checking the idle pool,
         * and removing sockets that have expired.
         */
        private static class IdleTimeOutThread extends Thread
        {
            private IdleTimeOutThread ()
            {
                super ("IdleTimeOutThread from " + Thread.currentThread ().getName ());
            }

            public void run ()
            {
                while (!isInterrupted ())
                {
                    try
                    {
                        Thread.sleep (1000);
                    }
                    catch (InterruptedException e)
                    {
                        // someone is trying to shut us down
                        break;
                    }

                    // Loop thru idle connections, closing ones that have
                    // expired.
                    synchronized (s_connections)
                    {
                        for (Iterator iter = s_connections.entrySet ().iterator (); iter.hasNext ();)
                        {
                            Map.Entry entry = (Map.Entry) iter.next ();
                            List idleConns = (List) entry.getValue ();

                            for (Iterator iterator = idleConns.iterator (); iterator.hasNext ();)
                            {
                                Connection c = (Connection) iterator.next ();
                                if (c.isExpired ())
                                {
                                    iterator.remove ();

                                    // Put the socket in a pool to close
                                    s_closePool.add (c.getSocket ());

                                    if (logger.isTraceEnabled ())
                                    {
                                        logger.trace ("Removed item from pool because it expired.\n" + c);
                                    }
                                }
                            }

                            if (idleConns.size () == 0)
                            {
                                // Don't want empty lists hanging around
                                iter.remove ();
                            }

                        }
                    }
                }
            }
        }
    } // end of inner class ConnectionPool

    // Generate unique request id's for Log messages
    private static int msgId = 0;

    private synchronized static int getMsgId ()
    {
        return ++msgId;
    }

    /** Class that describes a client provider for a given protocol */
    private static class Provider
    {
        private final static Class[] s_methodArgs = new Class[]
        { String.class };

        private final String m_name;
        private final Class m_class;
        private final Method m_factory;

        /*
         * Build a provider from the given class name. The provider must be the
         * name of a reachable class that extends net.pinfly.http.HttpClient and
         * that has a public getInstance method that takes a single String
         * argument and returns an class that extends HttpClient
         */
        Provider (String name) throws IllegalArgumentException, ClassNotFoundException, NoSuchMethodException
        {
            m_name = name;

            // Get the class
            m_class = Class.forName (name);

            // Get the newInstance factory method
            m_factory = m_class.getMethod ("getInstance", s_methodArgs);

            // Make sure the method is static
            if ((m_factory.getModifiers () & Modifier.STATIC) == 0)
            {
                throw new IllegalArgumentException ("getInstance() method in " + name + " is not static.");
            }

            // Verify that getInstance returns a child of
            // net.pinfly.http.HttpClient
            Class providerClass = m_factory.getReturnType ();
            if (!(HttpClient.class.isAssignableFrom (providerClass)))
            {
                throw new IllegalArgumentException ("Provider class " + m_name
                                                    + " is not a child of net.pinfly.http.HttpClient.");
            }
        }

        // For backward compatibility reasons, this method cannot throw
        // any checked exceptions.
        // It will throw an IllegalArgumentException if the underlying
        // constructor throws an exception.
        // It will throw a ProviderException if any other type of
        // error occurs.
        HttpClient getInstance (String url)
        {
            try
            {
                return (HttpClient) m_factory.invoke (null, new Object[]
                { url });
            }
            catch (InvocationTargetException ex)
            {
                // The underlying constructor threw an exception
                Throwable t = ex.getTargetException ();
                StringBuffer msg = new StringBuffer ();
                msg.append ("Error constructing instance of ").append (m_name).append (" for URL ").append (url);
                logger.warn (msg.toString (), t);
                throw new IllegalArgumentException (msg.toString ());
            }
            catch (Exception e)
            {
                StringBuffer msg = new StringBuffer ();
                msg.append ("Unexpected error constructing instance of ").append (m_name).append (" for URL ")
                   .append (url);
                logger.warn (msg.toString (), e);
                throw new ProviderException (msg.toString ());
            }

        }

        String getClassName ()
        {
            return m_name;
        }
    }

    // /** main
    // * Used for testing.
    // */
    // public static void main(String[] args)
    // {
    // try
    // {
    // HttpClient clnt = HttpClient.getHttpClient
    // ("http://phx-ricks:8765/cfx/cm/admin/support/showRequest.jsp?arg1=val1");
    // String response = clnt.get("arg2=val2");
    // System.out.println ("Got response of " + response.length() + " bytes");
    // response = clnt.post ("param1=value1", CONTENT_FORM_URLENCODED);
    // System.out.println ("Got response of " + response.length() + " bytes");
    //
    // HashMap map = new HashMap();
    // map.put("Field1", "Value1+3");
    // map.put("Field2", "Value2 asdf=234");
    // map.put("Field3", new String[]{"Value3a", "Value3b", "Value3c"});
    // map.put("Field blank", null);
    // response = clnt.post(map);
    // System.out.println ("Got response of " + response.length() + " bytes");
    //
    //
    // HttpClient m_client =
    // HttpClient.getHttpClient("http://phx-ricks/componentregistry/GetContextManager.jsp");
    // String response =
    // m_client.get("?method=GetCMServerByHost&clientName=PHX-RICKS&ssl=0");
    // System.out.println("******** First response: " + response.trim());
    // response = m_client.getResponse().trim();
    // System.out.println("******** Second response: " + response);
    // Thread.sleep (6000);
    // System.out.println("******** Third response:  " +
    // m_client.getResponse().trim());
    //
    // }
    // catch (Exception e)
    // {
    // e.printStackTrace();
    // }
    // }
    //
    // try
    // {
    // HttpClient client = null;
    // client = HttpClient.getHttpClient
    // ("http://localhost/registries/jsp/Echo.jsp");
    // client = new HttpClient (new URL
    // ("http://localhost/registries/jsp/Echo.jsp"));
    // client = new HttpClient (new URL
    // ("https://localhost/registries/jsp/Echo.jsp"));
    // client = new HttpClient ("http://localhost/registries/jsp/Echo.jsp");
    // client = new HttpClient ("https://localhost/registries/jsp/Echo.jsp");
    // client = new HttpsClient ("https://localhost/registries/jsp/Echo.jsp");
    // client = new HttpsClient ("https://banking.wellsfargo.com");
    // client = new HttpClient ("http://banking.wellsfargo.com");
    //
    // System.out.println (client.toString());
    // int numThreads = 4;
    // int numIterations = 30;
    // int maxWait = 250;
    // java.util.Random rand = new java.util.Random();
    //
    // Thread[] threads = new Thread[numThreads];
    // for (int i = 0; i < numThreads; i++)
    // {
    // threads[i] = new Thread (new Tester (client, numIterations, maxWait,
    // rand));
    // }
    // for (int i = 0; i < numThreads; i++)
    // {
    // threads[i].start();
    // try
    // {
    // Thread.currentThread().sleep(10);
    // }
    // catch (InterruptedException e)
    // {
    // }
    // }
    // for (int i = 0; i < numThreads; i++)
    // {
    // try
    // {
    // threads[i].join();
    // }
    // catch (InterruptedException ie)
    // {
    // }
    // }
    // }
    // catch (Exception ex)
    // {
    // System.out.println (ex.getClass().getName());
    // System.out.println (ex.getMessage());
    // ex.printStackTrace();
    // }
    // System.exit(0);
    // }
    //
    // private static class Tester implements Runnable
    // {
    // private HttpClient m_client;
    // private int m_iterations;
    // private int m_maxWait;
    // private java.util.Random m_random;
    //
    // private Tester (HttpClient c, int iterations, int wait, java.util.Random
    // rand)
    // {
    // m_client = c;
    // m_iterations = iterations;
    // m_maxWait = wait;
    // m_random = rand;
    // }
    //
    // public void run()
    // {
    // long s = System.currentTimeMillis();
    // int tw = 0;
    // for (int i = 0; i < m_iterations; i++)
    // {
    // int wait = m_random.nextInt (m_maxWait);
    // tw += wait;
    // String echo = getRandString (10);
    // String arg = "?echo=" + echo + "&wait=" + wait;
    // String resp = null;
    // try
    // {
    // resp = m_client.get (arg);
    // if (!resp.trim().equals (echo))
    // {
    // System.out.println ("Invalid response: " + echo + " != " + resp);
    // }
    // }
    // catch (Exception ex)
    // {
    // System.out.println (ex.getClass().getName());
    // System.out.println (ex.getMessage());
    // ex.printStackTrace();
    // }
    // }
    // long e = System.currentTimeMillis();
    // System.out.println ("Average request time = " +
    // ( (e - s - tw)/m_iterations) + " msec");
    // }
    //
    // private char[] chars = new char[]
    // {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
    // 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
    // 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
    // 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'
    // };
    //
    // private String getRandString (int len)
    // {
    // char[] str = new char[len];
    // for (int i = 0; i < len; i++)
    // {
    // str[i] = chars[m_random.nextInt (chars.length)];
    // }
    // return new String (str);
    // }
    // }

}
